package com.bigdata.collector.labels;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bigdata.collector.model.Page;
import com.bigdata.collector.util.HttpUtils;

public class Collector4Reuters extends AbstractCollector {

	private static Logger logger = Logger.getLogger(Collector4Reuters.class);

	public Collector4Reuters(String seed) {
		super(seed);
	}

	public static void main(String[] args) {
		String[] seeds = {
				"http://glossary.cn.reuters.com/wiki/category/word%E9%87%91%E8%9E%8D",
				"http://glossary.cn.reuters.com/wiki/category/word%E7%BC%A9%E7%95%A5%E8%AF%AD",
				"http://glossary.cn.reuters.com/wiki/category/word%E5%80%BA%E5%88%B8.html",
				"http://glossary.cn.reuters.com/wiki/category/word%E9%A3%8E%E9%99%A9%E6%8A%95%E8%B5%84%E5%92%8C%E6%96%B0%E8%82%A1%E5%8F%91%E8%A1%8C",
				"http://glossary.cn.reuters.com/wiki/category/word%E6%8A%80%E6%9C%AF%E5%88%86%E6%9E%90",
				"http://glossary.cn.reuters.com/wiki/category/word%E5%A4%96%E6%B1%87",
				"http://glossary.cn.reuters.com/wiki/category/word%E8%82%A1%E7%A5%A8",
				"http://glossary.cn.reuters.com/wiki/category/word%E5%9F%BA%E9%87%91",
				"http://glossary.cn.reuters.com/wiki/category/word%E6%9C%9F%E8%B4%A7%E6%9C%9F%E6%9D%83",
				"http://glossary.cn.reuters.com/wiki/category/word%E8%B4%A2%E5%8A%A1%E7%A8%8E%E6%94%B6" };
		for (String seed : seeds) {
			Collector4Reuters ct = new Collector4Reuters(seed);
			ct.run();
		}

	}

	public void crawler(String url) {
		try {
			int pageNum = 1;
			String firstPage = "";
			while (true) {
				String path = url + "/page" + pageNum + ".html";
				logger.info(path);
				Page page = HttpUtils.fetchHttpResponse(path, 5);
				String content = new String(page.content, "utf-8");
				if (firstPage.equals(content))
					break;
				if (firstPage.equals(""))
					firstPage = content;
				Document doc = Jsoup.parse(content);
				if (!extractTags(doc))
					break;
				pageNum++;
			}
		} catch (Exception e) {
			logger.error(ExceptionUtils.getStackTrace(e));
		}

	}

	@Override
	public boolean extractTags(Document doc) {
		Elements es = doc.getElementsByClass("glosIndexItem");
		if (es.size() == 0)
			return false;
		for (Element e : es) {
			String label = e.getElementsByTag("h4").text();
			String cat = e.getElementsByClass("glosCateName").get(0)
					.getElementsByTag("a").text();
			if (label.split(" ").length > 1) {
				save2File("reuters", label.substring(0, label.lastIndexOf(" "))
						+ "," + label.substring(label.lastIndexOf(" ") + 1),
						cat, true);
			} else {
				save2File("reuters", label, cat, true);
			}
		}
		return true;
	}

	@Override
	public void extractURLs(Document doc) {

	}
}
